This notebook processes level 2 images through the calwebb_image3 skymatch and resample steps and examines outputs for different sky method parameters.
Set up data path and image list file.
Set up association files.
Modify average backgrould level of input images.
Run skymatch step on images.
Run skymatch and resample for each skymethod parameter (local, global, match, global+match).
Testing other parameters (nclip, usigma, lsigma, lower, upper, skystat). (Still TBD.)
These steps are set up with simulated MIRI F560W data of the LMC astrometric field. The notebook removes the astrometric field and replaces the field with noise images to help the pipeline testers have a better visualization of what the pipeline is doing with each set of parameters.
This notebook has also been modified to test how skymatch and resample work together (with subtract=True indicating that the subtraction is done in the skymatch step and subtract=False indicating that the subtraction is done in the resample step). There are displays of the combined noise image to see how well the backgrounds were subtracted in the full combined image.
The pipeline documentation can be found here: https://jwst-pipeline.readthedocs.io/en/latest/jwst/skymatch/README.html
The pipeline code is available on GitHub: https://github.com/spacetelescope/jwst/tree/master/jwst/skymatch
Authors: T. Temim and M. Cracraft Last modified: 01/18/2022
# Create a temporary directory to hold notebook output, and change the working directory to that directory.
from tempfile import TemporaryDirectory
import os
data_dir = TemporaryDirectory()
os.chdir(data_dir.name)
# Set up CRDS options
import os
if "CRDS_CACHE_TYPE" in os.environ:
if os.environ['CRDS_CACHE_TYPE'] == 'local':
os.environ['CRDS_PATH'] = os.path.join(os.environ['HOME'], 'crds', 'cache')
elif os.path.isdir(os.environ['CRDS_CACHE_TYPE']):
os.environ['CRDS_PATH'] = os.environ['CRDS_CACHE_TYPE']
The following packages are needed to run this notebook:
import pytest
from astropy.io import fits
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
import jwst
import json
from jwst.skymatch import skymatch
from jwst.resample import ResampleStep
from jwst.pipeline import Image3Pipeline
from jwst.skymatch.skyimage import SkyImage
from jwst.associations.lib.rules_level3_base import DMS_Level3_Base
from jwst.associations import asn_from_list
from jwst import datamodels
from jwst.datamodels import ImageModel
from matplotlib import rcParams
from ci_watson.artifactory_helpers import get_bigdata
jwst.__version__
'1.8.2'
input_file1 = get_bigdata('jwst_validation_notebooks',
'validation_data',
'outlier_detection',
'outlier_detection_miri_test',
'det_image_seq1_MIRIMAGE_F560Wexp1_cal.fits')
input_file2 = get_bigdata('jwst_validation_notebooks',
'validation_data',
'outlier_detection',
'outlier_detection_miri_test',
'det_image_seq2_MIRIMAGE_F560Wexp1_cal.fits')
input_file3 = get_bigdata('jwst_validation_notebooks',
'validation_data',
'outlier_detection',
'outlier_detection_miri_test',
'det_image_seq3_MIRIMAGE_F560Wexp1_cal.fits')
input_file4 = get_bigdata('jwst_validation_notebooks',
'validation_data',
'outlier_detection',
'outlier_detection_miri_test',
'det_image_seq4_MIRIMAGE_F560Wexp1_cal.fits')
input_files=[]
input_files=[input_file1,input_file2,input_file3,input_file4]
imlist1=['det_image_seq1_MIRIMAGE_F560Wexp1_cal.fits','det_image_seq2_MIRIMAGE_F560Wexp1_cal.fits','det_image_seq3_MIRIMAGE_F560Wexp1_cal.fits','det_image_seq4_MIRIMAGE_F560Wexp1_cal.fits']
# Look at one image
im_file = ImageModel('det_image_seq1_MIRIMAGE_F560Wexp1_cal.fits')
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=5)
plt.colorbar()
<matplotlib.colorbar.Colorbar at 0x7fb35ab51c10>
The level three pipeline relies on an association file to specify which files are to be combined and provide the output file name.
# use asn_from_list to create association table
cal_list=imlist1
asn = asn_from_list.asn_from_list(cal_list, rule=DMS_Level3_Base, product_name='skymatch_combined.fits')
# use this if you need to add non'science' exposure types
#asn['products'][0]['members'][1]['exptype'] = 'background'
#asn['products'][0]['members'][2]['exptype'] = 'sourcecat'
# dump association table to a .json file for use in image3
with open('skymatch_asnfile.json', 'w') as fp:
fp.write(asn.dump()[1])
skymatch_json_file='skymatch_asnfile.json'
json_file = skymatch_json_file
file_list = []
file_list2 = []
with open(json_file) as json_data:
d = json.load(json_data)
members = d['products'][0]['members']
for item in np.arange(0,len(members)):
file_list.append(members[item]['expname'])
file_list2.append(members[item]['expname'][:-5]+"_skymatch.fits")
asn2 = asn_from_list.asn_from_list(file_list2, rule=DMS_Level3_Base, product_name=d['products'][0]['name'])
# use this if you need to add non'science' exposure types
#asn['products'][0]['members'][1]['exptype'] = 'background'
#asn['products'][0]['members'][2]['exptype'] = 'sourcecat'
# dump association table to a .json file for use in image3
with open('skymatch_asnfile2.json', 'w') as fp:
fp.write(asn2.dump()[1])
skymatch_json_file2='skymatch_asnfile2.json'
infile01_1 = input_files[0]
infile01_2 = input_files[1]
infile02_1 = input_files[2]
infile02_2 = input_files[3]
img01_1 = datamodels.open(infile01_1)
img01_2 = datamodels.open(infile01_2)
img02_1 = datamodels.open(infile02_1)
img02_2 = datamodels.open(infile02_2)
data01_1 = img01_1.data
data01_2 = img01_2.data
data02_1 = img02_1.data
data02_2 = img02_2.data
data01_1[data01_1<=0.3]=np.nan
data01_2[data01_2<=0.3]=np.nan
data02_1[data02_1<=0.3]=np.nan
data02_2[data02_2<=0.3]=np.nan
data01_1_orig = np.copy(img01_1.data)
data01_2_orig = np.copy(img01_2.data)
data02_1_orig = np.copy(img02_1.data)
data02_2_orig = np.copy(img02_2.data)
# check mean values of background
print('Mean:', np.mean(data01_1_orig[data01_1_orig<=4.0]),' Standard deviation: ',np.std(data01_1_orig[data01_1_orig<=4.0]))
print('Mean:', np.mean(data01_2_orig[data01_2_orig<=4.0]),' Standard deviation: ',np.std(data01_2_orig[data01_2_orig<=4.0]))
print('Mean:', np.mean(data02_1_orig[data02_1_orig<=4.0]),' Standard deviation: ',np.std(data02_1_orig[data02_1_orig<=4.0]))
print('Mean:', np.mean(data02_2_orig[data02_2_orig<=4.0]),' Standard deviation: ',np.std(data02_2_orig[data02_2_orig<=4.0]))
Mean: 0.93090874 Standard deviation: 0.1817127 Mean: 0.9362799 Standard deviation: 0.17500266 Mean: 0.9330347 Standard deviation: 0.17194116 Mean: 0.93216574 Standard deviation: 0.16481434
# creating a background image with specified mean and gaussian noise with sigma = 1.0
bkg_img_noise_neg2 = np.random.normal(-2,2*0.2,data01_1.shape)
bkg_img_noise_2 = np.random.normal(2,2*0.2,data01_1.shape)
bkg_img_noise_3 = np.random.normal(3,3*0.2,data01_1.shape)
bkg_img_noise_4 = np.random.normal(4,4*0.2,data01_1.shape)
bkg_img_noise_5 = np.random.normal(5,5*0.2,data01_1.shape)
bkg_img_noise_7 = np.random.normal(7,7*0.2,data01_1.shape)
Replace the images with a noise image for easier background visualization
# adding the new background with specified mean and gaussian noise (above) to image (This overwrites, not adds.)
img01_1.data=bkg_img_noise_2
img01_2.data=bkg_img_noise_3
img02_1.data=bkg_img_noise_5
img02_2.data=bkg_img_noise_2
# checking the mean and standard deviations of the new background values
print('Mean:',np.nanmean(img01_1.data),' Standard deviation: ',np.nanstd(img01_1.data))
print('Mean:',np.nanmean(img01_2.data),' Standard deviation: ',np.nanstd(img01_2.data))
print('Mean:',np.nanmean(img02_1.data),' Standard deviation: ',np.nanstd(img02_1.data))
print('Mean:',np.nanmean(img02_2.data),' Standard deviation: ',np.nanstd(img02_2.data))
Mean: 2.0005095 Standard deviation: 0.39995426 Mean: 2.9993074 Standard deviation: 0.60039955 Mean: 4.99976 Standard deviation: 1.0008261 Mean: 2.0005095 Standard deviation: 0.39995426
img01_1.save(file_list2[0],overwrite=True)
img01_2.save(file_list2[1],overwrite=True)
img02_1.save(file_list2[2],overwrite=True)
img02_2.save(file_list2[3],overwrite=True)
'det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits'
# Look at one image
im_file = ImageModel(file_list2[0])
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
<matplotlib.colorbar.Colorbar at 0x7fb358746700>
Run the pipeline on the data for different sets of parameters
Notes: ‘local’: compute sky background values of each input image or group of images (members of the same “exposure”). A single sky value is computed for each group of images. This method simply computes the mean/median/mode/etc. value of the “sky” separately in each input image. This will resulted in subtracted background being near zero in the combined image, as each image has it's individual background subtracted.
match_down specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True)
'Subtract' specifies whether the computed sky background values are to be subtracted from the images in the skymatch step. (Default = False) Currently, if not done in skymatch, the subtraction is performed in resample.
# skymatch, local, subtract= False, all 4 images used
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='local', subtract=False,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'local' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = False # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-12-04 05:09:42,747 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-12-04 05:09:42,748 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-12-04 05:09:42,750 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-12-04 05:09:42,752 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-12-04 05:09:42,753 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-12-04 05:09:42,754 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-12-04 05:09:42,756 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-12-04 05:09:42,889 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-12-04 05:09:42,896 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'local', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-12-04 05:09:43,043 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-12-04 05:09:43,294 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-12-04 05:09:43,297 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-12-04 05:09:43,950 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-12-04 05:09:43,953 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}
2022-12-04 05:09:43,953 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-12-04 05:09:43,963 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-12-04 05:09:44,142 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-12-04 05:09:44,143 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'local', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-12-04 05:09:44,273 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:09:44,274 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-12-04 05:09:44.273631
2022-12-04 05:09:44,274 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:09:44,275 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'local'
2022-12-04 05:09:44,275 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: OFF
2022-12-04 05:09:44,275 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:09:44,276 - stpipe.Image3Pipeline.skymatch - INFO - ---- Sky values computed per image and/or image groups.
2022-12-04 05:09:44,356 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953
2022-12-04 05:09:44,357 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.96494
2022-12-04 05:09:44,357 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 5.02361
2022-12-04 05:09:44,358 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953
2022-12-04 05:09:44,358 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:09:44,358 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-12-04 05:09:44.358397
2022-12-04 05:09:44,359 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:00.084766
2022-12-04 05:09:44,360 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:09:44,643 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-12-04 05:09:44,907 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-12-04 05:09:45,173 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-12-04 05:09:45,432 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-12-04 05:09:45,432 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-12-04 05:09:45,585 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-12-04 05:09:45,586 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}
2022-12-04 05:09:45,587 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-12-04 05:09:45,595 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-12-04 05:09:45,730 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-12-04 05:09:45,733 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}
2022-12-04 05:09:45,749 - stpipe.Image3Pipeline.resample - INFO - Using drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-12-04 05:09:45,771 - stpipe.Image3Pipeline.resample - INFO - Driz parameter kernel: square
2022-12-04 05:09:45,772 - stpipe.Image3Pipeline.resample - INFO - Driz parameter pixfrac: 1.0
2022-12-04 05:09:45,772 - stpipe.Image3Pipeline.resample - INFO - Driz parameter fillval: INDEF
2022-12-04 05:09:45,772 - stpipe.Image3Pipeline.resample - INFO - Driz parameter weight_type: ivm
2022-12-04 05:09:45,773 - stpipe.Image3Pipeline.resample - INFO - Output pixel scale ratio: 1.0
2022-12-04 05:09:45,903 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-12-04 05:09:46,540 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-12-04 05:09:47,150 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:47,924 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:48,865 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:49,648 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:49,863 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-12-04 05:09:50,505 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:51,301 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:52,102 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:52,914 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:53,121 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-12-04 05:09:53,735 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:54,569 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:55,379 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:56,166 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:56,365 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-12-04 05:09:56,956 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:57,748 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:58,525 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:59,315 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:09:59,546 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020232898 -0.018692220 0.023327823 0.016575172 359.989011943 0.019586596 359.985917018 -0.015680796
2022-12-04 05:10:00,011 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-12-04 05:10:00,012 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-12-04 05:10:00,203 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1149, 1118) from skymatch_combined_i2d.fits>,).
2022-12-04 05:10:00,205 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-12-04 05:10:00,206 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-12-04 05:10:00,209 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-12-04 05:10:00,210 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
# Check values out of skymatch step. With subtract = False, values should be equal.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('Original and new levels should be equal. If subtract=False, skymatch should not subtract.')
print('Mean: original, new (local)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
Original and new levels should be equal. If subtract=False, skymatch should not subtract. Mean: original, new (local) Mean: 2.0005095 , 2.0005095 Mean: 2.9993074 , 2.9993074 Mean: 4.99976 , 4.99976 Mean: 2.0005095 , 2.0005095
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('The resample step should subtract the individual backgrounds found in the skymatch step.')
print('Since the local individual background level is subtracted from each image, final expected result is near zero.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
The resample step should subtract the individual backgrounds found in the skymatch step. Since the local individual background level is subtracted from each image, final expected result is near zero. Mean: 0.014444155
<matplotlib.colorbar.Colorbar at 0x7fb352ec2190>
Skymatch step should subtract the background. Resample background level should match the skymatch background level since subtraction has already been done
# skymatch, local, subtract = True, all 4 images used
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='local', subtract=True,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'local' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
#match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = True # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
#pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-12-04 05:10:01,999 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-12-04 05:10:02,001 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-12-04 05:10:02,003 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-12-04 05:10:02,004 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-12-04 05:10:02,006 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-12-04 05:10:02,007 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-12-04 05:10:02,009 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-12-04 05:10:02,176 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-12-04 05:10:02,182 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'local', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-12-04 05:10:02,325 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-12-04 05:10:02,331 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-12-04 05:10:02,332 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-12-04 05:10:03,006 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-12-04 05:10:03,008 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}
2022-12-04 05:10:03,009 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-12-04 05:10:03,017 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-12-04 05:10:03,171 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-12-04 05:10:03,173 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'local', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-12-04 05:10:03,298 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:03,299 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-12-04 05:10:03.298536
2022-12-04 05:10:03,299 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:03,300 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'local'
2022-12-04 05:10:03,300 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: ON
2022-12-04 05:10:03,300 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:03,301 - stpipe.Image3Pipeline.skymatch - INFO - ---- Sky values computed per image and/or image groups.
2022-12-04 05:10:03,379 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953 (old=0, delta=1.98953)
2022-12-04 05:10:03,382 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.96494 (old=0, delta=2.96494)
2022-12-04 05:10:03,383 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 5.02361 (old=0, delta=5.02361)
2022-12-04 05:10:03,385 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953 (old=0, delta=1.98953)
2022-12-04 05:10:03,386 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:03,386 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-12-04 05:10:03.386130
2022-12-04 05:10:03,386 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:00.087594
2022-12-04 05:10:03,387 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:03,664 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-12-04 05:10:03,919 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-12-04 05:10:04,173 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-12-04 05:10:04,432 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-12-04 05:10:04,433 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-12-04 05:10:04,598 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-12-04 05:10:04,600 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}
2022-12-04 05:10:04,601 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-12-04 05:10:04,610 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-12-04 05:10:04,769 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-12-04 05:10:04,771 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}
2022-12-04 05:10:04,788 - stpipe.Image3Pipeline.resample - INFO - Using drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-12-04 05:10:04,808 - stpipe.Image3Pipeline.resample - INFO - Driz parameter kernel: square
2022-12-04 05:10:04,809 - stpipe.Image3Pipeline.resample - INFO - Driz parameter pixfrac: 1.0
2022-12-04 05:10:04,810 - stpipe.Image3Pipeline.resample - INFO - Driz parameter fillval: INDEF
2022-12-04 05:10:04,810 - stpipe.Image3Pipeline.resample - INFO - Driz parameter weight_type: ivm
2022-12-04 05:10:04,811 - stpipe.Image3Pipeline.resample - INFO - Output pixel scale ratio: 1.0
2022-12-04 05:10:04,943 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-12-04 05:10:05,580 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-12-04 05:10:06,189 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:06,957 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:07,743 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:08,551 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:08,733 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-12-04 05:10:09,322 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:10,104 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:10,896 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:11,689 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:11,887 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-12-04 05:10:12,489 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:13,294 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:14,087 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:14,888 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:15,094 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-12-04 05:10:15,683 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:16,466 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:17,223 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:17,990 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:18,211 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020232898 -0.018692220 0.023327823 0.016575172 359.989011943 0.019586596 359.985917018 -0.015680796
2022-12-04 05:10:18,685 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-12-04 05:10:18,686 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-12-04 05:10:18,900 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1149, 1118) from skymatch_combined_i2d.fits>,).
2022-12-04 05:10:18,901 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-12-04 05:10:18,902 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-12-04 05:10:18,905 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-12-04 05:10:18,905 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('With subtract=True, the new value should show that the background value was subtracted.')
print('Mean: original, new (local)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
With subtract=True, the new value should show that the background value was subtracted. Mean: original, new (local) Mean: 2.0005095 , 0.0109820105 Mean: 2.9993074 , 0.034370694 Mean: 4.99976 , -0.023850217 Mean: 2.0005095 , 0.0109820105
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('The background value was subtracted in the skymatch step, no additional subtraction should be done here.')
print('This value should match the values in the new column above.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
The background value was subtracted in the skymatch step, no additional subtraction should be done here. This value should match the values in the new column above. Mean: 0.014444155
<matplotlib.colorbar.Colorbar at 0x7fb35201f910>
This calculates an average sky across all images and subtracts that average from all images. There should be a warning for users for this step that makes it clear that this should not be used on images with differences in background levels.
# skymatch, local
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='local', subtract=False,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'global' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
#match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = False # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
#pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-12-04 05:10:20,650 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-12-04 05:10:20,651 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-12-04 05:10:20,653 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-12-04 05:10:20,654 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-12-04 05:10:20,655 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-12-04 05:10:20,657 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-12-04 05:10:20,658 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-12-04 05:10:20,880 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-12-04 05:10:20,886 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-12-04 05:10:21,024 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-12-04 05:10:21,026 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-12-04 05:10:21,028 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-12-04 05:10:21,695 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-12-04 05:10:21,697 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}
2022-12-04 05:10:21,698 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-12-04 05:10:21,706 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-12-04 05:10:21,875 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-12-04 05:10:21,877 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-12-04 05:10:22,004 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:22,004 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-12-04 05:10:22.004088
2022-12-04 05:10:22,005 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:22,005 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'global'
2022-12-04 05:10:22,006 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: OFF
2022-12-04 05:10:22,006 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:22,007 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing "global" sky - smallest sky value across *all* input images.
2022-12-04 05:10:22,084 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:22,085 - stpipe.Image3Pipeline.skymatch - INFO - "Global" sky value correction: 1.9895275424159538 [not converted]
2022-12-04 05:10:22,086 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953
2022-12-04 05:10:22,086 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953
2022-12-04 05:10:22,087 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953
2022-12-04 05:10:22,087 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953
2022-12-04 05:10:22,087 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:22,088 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-12-04 05:10:22.087970
2022-12-04 05:10:22,088 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:00.083882
2022-12-04 05:10:22,088 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:22,359 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-12-04 05:10:22,606 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-12-04 05:10:22,854 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-12-04 05:10:23,122 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-12-04 05:10:23,123 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-12-04 05:10:23,316 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-12-04 05:10:23,318 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}
2022-12-04 05:10:23,319 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-12-04 05:10:23,327 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-12-04 05:10:23,504 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-12-04 05:10:23,506 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}
2022-12-04 05:10:23,521 - stpipe.Image3Pipeline.resample - INFO - Using drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-12-04 05:10:23,543 - stpipe.Image3Pipeline.resample - INFO - Driz parameter kernel: square
2022-12-04 05:10:23,544 - stpipe.Image3Pipeline.resample - INFO - Driz parameter pixfrac: 1.0
2022-12-04 05:10:23,544 - stpipe.Image3Pipeline.resample - INFO - Driz parameter fillval: INDEF
2022-12-04 05:10:23,545 - stpipe.Image3Pipeline.resample - INFO - Driz parameter weight_type: ivm
2022-12-04 05:10:23,545 - stpipe.Image3Pipeline.resample - INFO - Output pixel scale ratio: 1.0
2022-12-04 05:10:23,682 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-12-04 05:10:24,315 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-12-04 05:10:24,952 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:25,737 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:26,511 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:27,301 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:27,487 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-12-04 05:10:28,060 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:28,882 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:29,729 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:30,558 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:30,756 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-12-04 05:10:31,360 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:32,127 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:32,894 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:33,684 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:33,901 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-12-04 05:10:34,529 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:35,296 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:36,085 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:36,861 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:37,087 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020232898 -0.018692220 0.023327823 0.016575172 359.989011943 0.019586596 359.985917018 -0.015680796
2022-12-04 05:10:37,543 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-12-04 05:10:37,543 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-12-04 05:10:37,774 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1149, 1118) from skymatch_combined_i2d.fits>,).
2022-12-04 05:10:37,775 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-12-04 05:10:37,776 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-12-04 05:10:37,779 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-12-04 05:10:37,779 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('There should be no subtraction at this step, with subtraction = False.')
print('Mean: original, new (local)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
There should be no subtraction at this step, with subtraction = False. Mean: original, new (local) Mean: 2.0005095 , 2.0005095 Mean: 2.9993074 , 2.9993074 Mean: 4.99976 , 4.99976 Mean: 2.0005095 , 2.0005095
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('Global finds a single value for the background, and subtracts that value from all images.')
print('The value listed here should be the average of the subtracted sky values across images.')
print('The combined image will look messy as the same value was subtracted from the image with background = 5')
print('and from the images with background = 2. This shows the differing backgrounds strongly.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
Global finds a single value for the background, and subtracts that value from all images. The value listed here should be the average of the subtracted sky values across images. The combined image will look messy as the same value was subtracted from the image with background = 5 and from the images with background = 2. This shows the differing backgrounds strongly. Mean: 1.0169634
<matplotlib.colorbar.Colorbar at 0x7fb35154d5b0>
# skymatch, global, subtract = True
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='global', subtract=True,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'global' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
#match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = True # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
#pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-12-04 05:10:39,556 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-12-04 05:10:39,558 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-12-04 05:10:39,560 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-12-04 05:10:39,561 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-12-04 05:10:39,562 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-12-04 05:10:39,564 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-12-04 05:10:39,565 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-12-04 05:10:39,802 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-12-04 05:10:39,809 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-12-04 05:10:39,953 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-12-04 05:10:39,955 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-12-04 05:10:39,956 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-12-04 05:10:40,650 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-12-04 05:10:40,652 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}
2022-12-04 05:10:40,652 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-12-04 05:10:40,662 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-12-04 05:10:40,845 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-12-04 05:10:40,846 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-12-04 05:10:40,971 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:40,972 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-12-04 05:10:40.971828
2022-12-04 05:10:40,973 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:40,973 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'global'
2022-12-04 05:10:40,973 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: ON
2022-12-04 05:10:40,974 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:40,974 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing "global" sky - smallest sky value across *all* input images.
2022-12-04 05:10:41,052 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:41,053 - stpipe.Image3Pipeline.skymatch - INFO - "Global" sky value correction: 1.9895275424159538 [not converted]
2022-12-04 05:10:41,055 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953 (old=0, delta=1.98953)
2022-12-04 05:10:41,057 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953 (old=0, delta=1.98953)
2022-12-04 05:10:41,059 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953 (old=0, delta=1.98953)
2022-12-04 05:10:41,060 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953 (old=0, delta=1.98953)
2022-12-04 05:10:41,061 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:41,061 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-12-04 05:10:41.061370
2022-12-04 05:10:41,062 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:00.089542
2022-12-04 05:10:41,062 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:41,337 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-12-04 05:10:41,593 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-12-04 05:10:41,844 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-12-04 05:10:42,092 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-12-04 05:10:42,093 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-12-04 05:10:42,290 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-12-04 05:10:42,292 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}
2022-12-04 05:10:42,293 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-12-04 05:10:42,301 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-12-04 05:10:42,488 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-12-04 05:10:42,490 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}
2022-12-04 05:10:42,506 - stpipe.Image3Pipeline.resample - INFO - Using drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-12-04 05:10:42,526 - stpipe.Image3Pipeline.resample - INFO - Driz parameter kernel: square
2022-12-04 05:10:42,526 - stpipe.Image3Pipeline.resample - INFO - Driz parameter pixfrac: 1.0
2022-12-04 05:10:42,527 - stpipe.Image3Pipeline.resample - INFO - Driz parameter fillval: INDEF
2022-12-04 05:10:42,527 - stpipe.Image3Pipeline.resample - INFO - Driz parameter weight_type: ivm
2022-12-04 05:10:42,527 - stpipe.Image3Pipeline.resample - INFO - Output pixel scale ratio: 1.0
2022-12-04 05:10:42,660 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-12-04 05:10:43,297 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-12-04 05:10:43,883 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:44,668 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:45,454 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:46,238 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:46,440 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-12-04 05:10:47,045 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:47,856 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:48,655 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:49,452 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:49,646 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-12-04 05:10:50,243 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:51,018 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:51,824 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:52,603 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:52,807 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-12-04 05:10:53,374 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:54,131 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:54,885 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:55,667 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:10:55,883 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020232898 -0.018692220 0.023327823 0.016575172 359.989011943 0.019586596 359.985917018 -0.015680796
2022-12-04 05:10:56,339 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-12-04 05:10:56,340 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-12-04 05:10:56,560 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1149, 1118) from skymatch_combined_i2d.fits>,).
2022-12-04 05:10:56,561 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-12-04 05:10:56,562 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-12-04 05:10:56,565 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-12-04 05:10:56,566 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('Global finds a single value for the background, and subtracts that value from all images.')
print('The values listed here should be the average of the subtracted sky values across images subtracted from each image.')
print('Mean: original, new (global)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
Global finds a single value for the background, and subtracts that value from all images. The values listed here should be the average of the subtracted sky values across images subtracted from each image. Mean: original, new (global) Mean: 2.0005095 , 0.0109820105 Mean: 2.9993074 , 1.009779 Mean: 4.99976 , 3.0102339 Mean: 2.0005095 , 0.0109820105
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('Global finds a single value for the background, and subtracts that value from all images.')
print('The value listed here should be the average of the subtracted sky values across images.')
print('The combined image will look messy as the same value was subtracted from the image with background = 5')
print('and from the images with background = 2. This shows the differing backgrounds strongly.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
Global finds a single value for the background, and subtracts that value from all images. The value listed here should be the average of the subtracted sky values across images. The combined image will look messy as the same value was subtracted from the image with background = 5 and from the images with background = 2. This shows the differing backgrounds strongly. Mean: 1.0169634
<matplotlib.colorbar.Colorbar at 0x7fb3510386a0>
Based on whether match_down is set to True or False, Match will calculate the difference between the lowest background level or highest level (respectively), and subtract the difference between the calculated level and the 'matched' level.
This is the preferred default option as it subtracts the differences between the background levels, normalizing all of the backgrounds to a common level (either lowest or highest background), rather than subtracting off all of the background.
# skymatch, match down, subtract = True
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='match', match_down=True,subtract=True,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'match' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = True # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-12-04 05:10:58,288 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-12-04 05:10:58,290 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-12-04 05:10:58,291 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-12-04 05:10:58,293 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-12-04 05:10:58,294 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-12-04 05:10:58,295 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-12-04 05:10:58,296 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-12-04 05:10:58,546 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-12-04 05:10:58,553 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-12-04 05:10:58,690 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-12-04 05:10:58,695 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-12-04 05:10:58,697 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-12-04 05:10:59,403 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-12-04 05:10:59,405 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}
2022-12-04 05:10:59,406 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-12-04 05:10:59,415 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-12-04 05:10:59,603 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-12-04 05:10:59,605 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-12-04 05:10:59,732 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:59,733 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-12-04 05:10:59.732704
2022-12-04 05:10:59,733 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:59,734 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'match'
2022-12-04 05:10:59,734 - stpipe.Image3Pipeline.skymatch - INFO - Sky matching direction: DOWN
2022-12-04 05:10:59,734 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: ON
2022-12-04 05:10:59,735 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:10:59,735 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing differences in sky values in overlapping regions.
2022-12-04 05:11:02,200 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.00522893 (old=0, delta=0.00522893)
2022-12-04 05:11:02,202 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.979458 (old=0, delta=0.979458)
2022-12-04 05:11:02,204 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 3.03167 (old=0, delta=3.03167)
2022-12-04 05:11:02,206 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0 (old=0, delta=0)
2022-12-04 05:11:02,206 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:02,207 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-12-04 05:11:02.206883
2022-12-04 05:11:02,207 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:02.474179
2022-12-04 05:11:02,207 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:02,475 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-12-04 05:11:02,721 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-12-04 05:11:02,971 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-12-04 05:11:03,221 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-12-04 05:11:03,222 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-12-04 05:11:03,422 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-12-04 05:11:03,423 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}
2022-12-04 05:11:03,424 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-12-04 05:11:03,432 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-12-04 05:11:03,620 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-12-04 05:11:03,622 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}
2022-12-04 05:11:03,640 - stpipe.Image3Pipeline.resample - INFO - Using drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-12-04 05:11:03,660 - stpipe.Image3Pipeline.resample - INFO - Driz parameter kernel: square
2022-12-04 05:11:03,661 - stpipe.Image3Pipeline.resample - INFO - Driz parameter pixfrac: 1.0
2022-12-04 05:11:03,661 - stpipe.Image3Pipeline.resample - INFO - Driz parameter fillval: INDEF
2022-12-04 05:11:03,661 - stpipe.Image3Pipeline.resample - INFO - Driz parameter weight_type: ivm
2022-12-04 05:11:03,662 - stpipe.Image3Pipeline.resample - INFO - Output pixel scale ratio: 1.0
2022-12-04 05:11:03,796 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-12-04 05:11:04,431 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-12-04 05:11:05,010 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:05,799 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:06,607 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:07,401 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:07,619 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-12-04 05:11:08,215 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:09,024 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:09,800 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:10,571 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:10,775 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-12-04 05:11:11,400 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:12,182 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:12,945 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:13,712 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:13,906 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-12-04 05:11:14,504 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:15,263 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:16,027 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:16,830 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:17,058 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020232898 -0.018692220 0.023327823 0.016575172 359.989011943 0.019586596 359.985917018 -0.015680796
2022-12-04 05:11:17,513 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-12-04 05:11:17,514 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-12-04 05:11:17,721 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1149, 1118) from skymatch_combined_i2d.fits>,).
2022-12-04 05:11:17,723 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-12-04 05:11:17,723 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-12-04 05:11:17,726 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-12-04 05:11:17,727 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('There should be no subtraction at this step, with subtraction = False.')
print('Mean: original, new (match down)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
There should be no subtraction at this step, with subtraction = False. Mean: original, new (match down) Mean: 2.0005095 , 1.9952799 Mean: 2.9993074 , 2.0198486 Mean: 4.99976 , 1.9680889 Mean: 2.0005095 , 2.0005095
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('For the method match with match_down= True, the value of the background should match the minimum of the individual backgrounds.')
print('This value should match the values in the new column above. Subtraction was done in skymatch step.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
For the method match with match_down= True, the value of the background should match the minimum of the individual backgrounds. This value should match the values in the new column above. Subtraction was done in skymatch step. Mean: 2.0022504
<matplotlib.colorbar.Colorbar at 0x7fb350b80850>
# skymatch, match up, subtract = True
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='match', match_down=False,subtract=True,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'match' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
match_down = False # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = True # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-12-04 05:11:19,474 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-12-04 05:11:19,476 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-12-04 05:11:19,477 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-12-04 05:11:19,479 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-12-04 05:11:19,480 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-12-04 05:11:19,481 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-12-04 05:11:19,483 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-12-04 05:11:19,747 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-12-04 05:11:19,753 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': False, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-12-04 05:11:19,895 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-12-04 05:11:19,898 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-12-04 05:11:19,899 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-12-04 05:11:20,624 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-12-04 05:11:20,627 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}
2022-12-04 05:11:20,628 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-12-04 05:11:20,636 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-12-04 05:11:20,841 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-12-04 05:11:20,843 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': False, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-12-04 05:11:20,970 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:20,971 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-12-04 05:11:20.970588
2022-12-04 05:11:20,972 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:20,972 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'match'
2022-12-04 05:11:20,973 - stpipe.Image3Pipeline.skymatch - INFO - Sky matching direction: UP
2022-12-04 05:11:20,973 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: ON
2022-12-04 05:11:20,974 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:20,974 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing differences in sky values in overlapping regions.
2022-12-04 05:11:23,488 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: -3.02644 (old=0, delta=-3.02644)
2022-12-04 05:11:23,491 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: -2.05221 (old=0, delta=-2.05221)
2022-12-04 05:11:23,493 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0 (old=0, delta=0)
2022-12-04 05:11:23,495 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: -3.03167 (old=0, delta=-3.03167)
2022-12-04 05:11:23,495 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:23,496 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-12-04 05:11:23.495978
2022-12-04 05:11:23,496 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:02.525390
2022-12-04 05:11:23,497 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:23,778 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-12-04 05:11:24,037 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-12-04 05:11:24,290 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-12-04 05:11:24,537 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-12-04 05:11:24,538 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-12-04 05:11:24,757 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-12-04 05:11:24,759 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}
2022-12-04 05:11:24,760 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-12-04 05:11:24,769 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-12-04 05:11:24,987 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-12-04 05:11:24,989 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}
2022-12-04 05:11:25,007 - stpipe.Image3Pipeline.resample - INFO - Using drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-12-04 05:11:25,029 - stpipe.Image3Pipeline.resample - INFO - Driz parameter kernel: square
2022-12-04 05:11:25,030 - stpipe.Image3Pipeline.resample - INFO - Driz parameter pixfrac: 1.0
2022-12-04 05:11:25,030 - stpipe.Image3Pipeline.resample - INFO - Driz parameter fillval: INDEF
2022-12-04 05:11:25,031 - stpipe.Image3Pipeline.resample - INFO - Driz parameter weight_type: ivm
2022-12-04 05:11:25,031 - stpipe.Image3Pipeline.resample - INFO - Output pixel scale ratio: 1.0
2022-12-04 05:11:25,167 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-12-04 05:11:25,805 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-12-04 05:11:26,410 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:27,188 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:27,984 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:28,773 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:28,961 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-12-04 05:11:29,567 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:30,357 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:31,123 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:31,914 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:32,112 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-12-04 05:11:32,688 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:33,471 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:34,247 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:35,008 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:35,200 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-12-04 05:11:35,776 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:36,565 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:37,324 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:38,111 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:38,333 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020232898 -0.018692220 0.023327823 0.016575172 359.989011943 0.019586596 359.985917018 -0.015680796
2022-12-04 05:11:38,789 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-12-04 05:11:38,791 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-12-04 05:11:39,040 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1149, 1118) from skymatch_combined_i2d.fits>,).
2022-12-04 05:11:39,042 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-12-04 05:11:39,042 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-12-04 05:11:39,046 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-12-04 05:11:39,048 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('With match_down = False, match will match the backgrounds of all to the highest background.')
print('With subtract = True, the background matching is done at the skymatch step.')
print('Mean: original, new (match up)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
With match_down = False, match will match the backgrounds of all to the highest background. With subtract = True, the background matching is done at the skymatch step. Mean: original, new (match up) Mean: 2.0005095 , 5.0269537 Mean: 2.9993074 , 5.051522 Mean: 4.99976 , 4.99976 Mean: 2.0005095 , 5.0321817
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('For the method match with match_down=False, the value of the background should match the maximum of the individual backgrounds')
print('This value should match the values in the new column above. Subtraction was done in skymatch step.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
For the method match with match_down=False, the value of the background should match the maximum of the individual backgrounds This value should match the values in the new column above. Subtraction was done in skymatch step. Mean: 5.0339227
<matplotlib.colorbar.Colorbar at 0x7fb3507f4be0>
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'match' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = False # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-12-04 05:11:40,461 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-12-04 05:11:40,462 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-12-04 05:11:40,464 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-12-04 05:11:40,466 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-12-04 05:11:40,467 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-12-04 05:11:40,469 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-12-04 05:11:40,470 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-12-04 05:11:40,739 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-12-04 05:11:40,746 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-12-04 05:11:40,914 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-12-04 05:11:40,916 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-12-04 05:11:40,919 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-12-04 05:11:41,988 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-12-04 05:11:41,991 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}
2022-12-04 05:11:41,991 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-12-04 05:11:42,001 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-12-04 05:11:42,504 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-12-04 05:11:42,507 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-12-04 05:11:42,666 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:42,667 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-12-04 05:11:42.666906
2022-12-04 05:11:42,668 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:42,668 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'match'
2022-12-04 05:11:42,669 - stpipe.Image3Pipeline.skymatch - INFO - Sky matching direction: DOWN
2022-12-04 05:11:42,669 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: OFF
2022-12-04 05:11:42,669 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:42,670 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing differences in sky values in overlapping regions.
2022-12-04 05:11:45,256 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.00522893
2022-12-04 05:11:45,257 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.979458
2022-12-04 05:11:45,258 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 3.03167
2022-12-04 05:11:45,258 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0
2022-12-04 05:11:45,259 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:45,259 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-12-04 05:11:45.259165
2022-12-04 05:11:45,259 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:02.592259
2022-12-04 05:11:45,260 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:11:45,599 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-12-04 05:11:45,893 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-12-04 05:11:46,195 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-12-04 05:11:46,503 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-12-04 05:11:46,504 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-12-04 05:11:47,038 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-12-04 05:11:47,041 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}
2022-12-04 05:11:47,042 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-12-04 05:11:47,051 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-12-04 05:11:47,555 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-12-04 05:11:47,558 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}
2022-12-04 05:11:47,579 - stpipe.Image3Pipeline.resample - INFO - Using drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-12-04 05:11:47,604 - stpipe.Image3Pipeline.resample - INFO - Driz parameter kernel: square
2022-12-04 05:11:47,604 - stpipe.Image3Pipeline.resample - INFO - Driz parameter pixfrac: 1.0
2022-12-04 05:11:47,605 - stpipe.Image3Pipeline.resample - INFO - Driz parameter fillval: INDEF
2022-12-04 05:11:47,605 - stpipe.Image3Pipeline.resample - INFO - Driz parameter weight_type: ivm
2022-12-04 05:11:47,606 - stpipe.Image3Pipeline.resample - INFO - Output pixel scale ratio: 1.0
2022-12-04 05:11:47,766 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-12-04 05:11:48,485 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-12-04 05:11:49,165 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:50,013 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:50,828 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:51,688 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:51,874 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-12-04 05:11:52,459 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:53,262 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:54,049 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:54,838 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:55,052 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-12-04 05:11:55,670 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:56,499 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:57,331 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:58,178 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:58,385 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-12-04 05:11:58,995 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:11:59,814 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:00,601 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:01,411 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:01,635 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020232898 -0.018692220 0.023327823 0.016575172 359.989011943 0.019586596 359.985917018 -0.015680796
2022-12-04 05:12:02,110 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-12-04 05:12:02,111 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-12-04 05:12:02,429 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1149, 1118) from skymatch_combined_i2d.fits>,).
2022-12-04 05:12:02,431 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-12-04 05:12:02,432 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-12-04 05:12:02,435 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-12-04 05:12:02,436 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('With subtract=False, no subtraction should be done here.')
print('Mean: original, new (match down)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
With subtract=False, no subtraction should be done here. Mean: original, new (match down) Mean: 2.0005095 , 2.0005095 Mean: 2.9993074 , 2.9993074 Mean: 4.99976 , 4.99976 Mean: 2.0005095 , 2.0005095
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('With the match method, subtract=False and match_down=True, the background value should match to the lowest individual background.')
print('This value should match the minimum values in the new column above.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
With the match method, subtract=False and match_down=True, the background value should match to the lowest individual background. This value should match the minimum values in the new column above. Mean: 2.0022504
<matplotlib.colorbar.Colorbar at 0x7fb3503befa0>
The behavior of this step is that it subtracts all background. More documentation is needed on how exactly it gets to that point.
# skymatch, global+match, subtract = True
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='global+match', match_down=True,subtract=True,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'global+match' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
#match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = True # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
#pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-12-04 05:12:04,254 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-12-04 05:12:04,255 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-12-04 05:12:04,257 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-12-04 05:12:04,258 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-12-04 05:12:04,260 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-12-04 05:12:04,261 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-12-04 05:12:04,262 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-12-04 05:12:04,599 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-12-04 05:12:04,606 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global+match', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-12-04 05:12:04,744 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-12-04 05:12:04,750 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-12-04 05:12:04,752 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-12-04 05:12:05,476 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-12-04 05:12:05,479 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}
2022-12-04 05:12:05,479 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-12-04 05:12:05,488 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-12-04 05:12:05,700 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-12-04 05:12:05,702 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global+match', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-12-04 05:12:05,834 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:05,835 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-12-04 05:12:05.834434
2022-12-04 05:12:05,835 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:05,836 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'global+match'
2022-12-04 05:12:05,836 - stpipe.Image3Pipeline.skymatch - INFO - Sky matching direction: DOWN
2022-12-04 05:12:05,837 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: ON
2022-12-04 05:12:05,837 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:05,837 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing differences in sky values in overlapping regions.
2022-12-04 05:12:08,293 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.00522893 (old=0, delta=0.00522893)
2022-12-04 05:12:08,296 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.979458 (old=0, delta=0.979458)
2022-12-04 05:12:08,298 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 3.03167 (old=0, delta=3.03167)
2022-12-04 05:12:08,300 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0 (old=0, delta=0)
2022-12-04 05:12:08,301 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:08,301 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing "global" sky - smallest sky value across *all* input images.
2022-12-04 05:12:08,378 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:08,379 - stpipe.Image3Pipeline.skymatch - INFO - "Global" sky value correction: 1.9842953652073356 [not converted]
2022-12-04 05:12:08,379 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:08,379 - stpipe.Image3Pipeline.skymatch - INFO - ---- Final (match+global) sky for:
2022-12-04 05:12:08,381 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98952 (old=0.00522893, delta=1.9843)
2022-12-04 05:12:08,382 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.96375 (old=0.979458, delta=1.9843)
2022-12-04 05:12:08,383 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 5.01597 (old=3.03167, delta=1.9843)
2022-12-04 05:12:08,384 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.9843 (old=0, delta=1.9843)
2022-12-04 05:12:08,385 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:08,385 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-12-04 05:12:08.385041
2022-12-04 05:12:08,385 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:02.550607
2022-12-04 05:12:08,386 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:08,673 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-12-04 05:12:08,927 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-12-04 05:12:09,176 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-12-04 05:12:09,429 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-12-04 05:12:09,430 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-12-04 05:12:09,710 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-12-04 05:12:09,712 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}
2022-12-04 05:12:09,713 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-12-04 05:12:09,721 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-12-04 05:12:09,939 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-12-04 05:12:09,940 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}
2022-12-04 05:12:09,957 - stpipe.Image3Pipeline.resample - INFO - Using drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-12-04 05:12:09,979 - stpipe.Image3Pipeline.resample - INFO - Driz parameter kernel: square
2022-12-04 05:12:09,980 - stpipe.Image3Pipeline.resample - INFO - Driz parameter pixfrac: 1.0
2022-12-04 05:12:09,980 - stpipe.Image3Pipeline.resample - INFO - Driz parameter fillval: INDEF
2022-12-04 05:12:09,980 - stpipe.Image3Pipeline.resample - INFO - Driz parameter weight_type: ivm
2022-12-04 05:12:09,981 - stpipe.Image3Pipeline.resample - INFO - Output pixel scale ratio: 1.0
2022-12-04 05:12:10,112 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-12-04 05:12:10,755 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-12-04 05:12:11,335 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:12,087 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:12,854 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:13,621 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:13,803 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-12-04 05:12:14,389 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:15,147 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:15,900 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:16,669 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:16,863 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-12-04 05:12:17,463 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:18,248 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:19,041 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:19,828 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:20,035 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-12-04 05:12:20,627 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:21,382 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:22,151 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:22,918 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:23,143 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020232898 -0.018692220 0.023327823 0.016575172 359.989011943 0.019586596 359.985917018 -0.015680796
2022-12-04 05:12:23,610 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-12-04 05:12:23,611 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-12-04 05:12:23,902 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1149, 1118) from skymatch_combined_i2d.fits>,).
2022-12-04 05:12:23,904 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-12-04 05:12:23,904 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-12-04 05:12:23,907 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-12-04 05:12:23,908 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('Mean: original, new (global+match)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
Mean: original, new (global+match) Mean: 2.0005095 , 0.010985287 Mean: 2.9993074 , 0.035552774 Mean: 4.99976 , -0.016206518 Mean: 2.0005095 , 0.016214222
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
#print('Did resample subtract a background level, and what was it?')
print('This value should match the values in the new column above. Subtraction was done in skymatch step.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
This value should match the values in the new column above. Subtraction was done in skymatch step. Mean: 0.017955
<matplotlib.colorbar.Colorbar at 0x7fb34ffd2700>
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'global+match' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
#match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = False # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
#pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-12-04 05:12:25,644 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-12-04 05:12:25,646 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-12-04 05:12:25,648 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-12-04 05:12:25,649 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-12-04 05:12:25,651 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-12-04 05:12:25,652 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-12-04 05:12:25,653 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-12-04 05:12:25,965 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-12-04 05:12:25,972 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global+match', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-12-04 05:12:26,113 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-12-04 05:12:26,117 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-12-04 05:12:26,119 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-12-04 05:12:26,862 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-12-04 05:12:26,864 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'use_custom_catalogs': False, 'catalog_format': 'ecsv', 'catfile': '', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}
2022-12-04 05:12:26,864 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-12-04 05:12:26,873 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-12-04 05:12:27,098 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-12-04 05:12:27,099 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpb6891b61/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global+match', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-12-04 05:12:27,229 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:27,230 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-12-04 05:12:27.229419
2022-12-04 05:12:27,230 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:27,230 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'global+match'
2022-12-04 05:12:27,231 - stpipe.Image3Pipeline.skymatch - INFO - Sky matching direction: DOWN
2022-12-04 05:12:27,231 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: OFF
2022-12-04 05:12:27,231 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:27,232 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing differences in sky values in overlapping regions.
2022-12-04 05:12:29,754 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.00522893
2022-12-04 05:12:29,754 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.979458
2022-12-04 05:12:29,755 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 3.03167
2022-12-04 05:12:29,755 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0
2022-12-04 05:12:29,756 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:29,756 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing "global" sky - smallest sky value across *all* input images.
2022-12-04 05:12:29,836 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:29,837 - stpipe.Image3Pipeline.skymatch - INFO - "Global" sky value correction: 1.9842986085316623 [not converted]
2022-12-04 05:12:29,837 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:29,837 - stpipe.Image3Pipeline.skymatch - INFO - ---- Final (match+global) sky for:
2022-12-04 05:12:29,838 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98953 (old=0.00522893, delta=1.9843)
2022-12-04 05:12:29,838 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.96376 (old=0.979458, delta=1.9843)
2022-12-04 05:12:29,838 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 5.01597 (old=3.03167, delta=1.9843)
2022-12-04 05:12:29,839 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.9843 (old=0, delta=1.9843)
2022-12-04 05:12:29,839 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:29,839 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-12-04 05:12:29.839663
2022-12-04 05:12:29,840 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:02.610244
2022-12-04 05:12:29,840 - stpipe.Image3Pipeline.skymatch - INFO -
2022-12-04 05:12:30,126 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-12-04 05:12:30,381 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-12-04 05:12:30,632 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-12-04 05:12:30,882 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-12-04 05:12:30,883 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-12-04 05:12:31,122 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-12-04 05:12:31,124 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}
2022-12-04 05:12:31,125 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-12-04 05:12:31,133 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-12-04 05:12:31,365 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-12-04 05:12:31,367 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}
2022-12-04 05:12:31,384 - stpipe.Image3Pipeline.resample - INFO - Using drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-12-04 05:12:31,405 - stpipe.Image3Pipeline.resample - INFO - Driz parameter kernel: square
2022-12-04 05:12:31,406 - stpipe.Image3Pipeline.resample - INFO - Driz parameter pixfrac: 1.0
2022-12-04 05:12:31,406 - stpipe.Image3Pipeline.resample - INFO - Driz parameter fillval: INDEF
2022-12-04 05:12:31,407 - stpipe.Image3Pipeline.resample - INFO - Driz parameter weight_type: ivm
2022-12-04 05:12:31,407 - stpipe.Image3Pipeline.resample - INFO - Output pixel scale ratio: 1.0
2022-12-04 05:12:31,540 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-12-04 05:12:32,183 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-12-04 05:12:32,822 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:33,606 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:34,361 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:35,100 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:35,281 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-12-04 05:12:35,854 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:36,627 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:37,413 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:38,178 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:38,374 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-12-04 05:12:38,961 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:39,724 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:40,486 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:41,279 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:41,474 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-12-04 05:12:42,045 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:42,794 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:43,545 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:44,326 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1149, 1118)
2022-12-04 05:12:44,546 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020232898 -0.018692220 0.023327823 0.016575172 359.989011943 0.019586596 359.985917018 -0.015680796
2022-12-04 05:12:45,013 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-12-04 05:12:45,014 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-12-04 05:12:45,300 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1149, 1118) from skymatch_combined_i2d.fits>,).
2022-12-04 05:12:45,302 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-12-04 05:12:45,302 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-12-04 05:12:45,305 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-12-04 05:12:45,306 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('Mean: original, new (global+match)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
Mean: original, new (global+match) Mean: 2.0005095 , 2.0005095 Mean: 2.9993074 , 2.9993074 Mean: 4.99976 , 4.99976 Mean: 2.0005095 , 2.0005095
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('Did resample subtract a background level, and what was it?')
#print('This value should match the values in the new column above. Subtraction was done in skymatch step.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
Did resample subtract a background level, and what was it? Mean: 0.017951768
<matplotlib.colorbar.Colorbar at 0x7fb34fb93af0>
